package com.tledu.spider;

import com.tledu.pageprocessor.QNPageProcessor;
import com.tledu.pipeline.NewsPipeline;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.Spider;

import java.util.HashMap;
import java.util.Map;

public class QNSpider {
    public static void main(String[] args) {
        Request request = new Request();
        request.setUrl("http://news.youth.cn/gn/index.htm");
        Map<String, Object> extras = new HashMap<>();
        extras.put("index", 0);
        request.setExtras(extras);

        Spider.create(new QNPageProcessor())
                // 出了添加Url之外，也可以添加Request对象
                .addRequest(request)
                .addPipeline(new NewsPipeline())
                .thread(5)
                .run();
    }
}
